% generate_synthetic_unemployed_data.m
% Generates synthetic negative class (unemployed) data and extracts features
% Input: Excel file '附件1 数据-C题.xlsx'
% Output: Synthetic negative class data saved as '负类训练数据.xlsx'
%         Processed data with extracted features saved as '负类训练数据_特征提取.xlsx'

% Clear workspace
clear all;
clc;

% Read the Excel file
data = readtable('附件1 数据-C题.xlsx', 'TreatAsEmpty', {'\N', ''});

% Identify negative class (unemployed) samples
% Assume unemployed if c_acc020 (unemployment registration number) or c_ajc090 (unemployment time) is non-empty
% and b_acc031 (employment time) is empty or in the past
data.is_unemployed = ~ismissing(data.c_acc020) | ~ismissing(data.c_ajc090);

% Handle b_acc031 date conversion safely
data.is_employed = false(height(data), 1); % Initialize as false
valid_dates = ~ismissing(data.b_acc031); % Non-missing entries
try
    % Convert valid b_acc031 entries to datetime with specified format
    emp_dates = datetime(data.b_acc031(valid_dates), ...
        'InputFormat', 'yyyy-MM-dd HH:mm:ss', ...
        'Locale', 'zh_CN');
    % Check if employment date is recent (>= 2020-01-01)
    data.is_employed(valid_dates) = emp_dates >= datetime('2020-01-01');
catch e
    warning('Error converting b_acc031 dates: %s', e.message);
    % If conversion fails, treat all as unemployed for safety
    data.is_employed(:) = false;
end

% Select negative class (unemployed and not employed)
negative_class = data(data.is_unemployed & ~data.is_employed, :);

% Calculate number of synthetic samples needed
% Current negative class proportion is 20%, aim for 50%
total_samples = height(data);
neg_samples = height(negative_class);
target_neg_proportion = 0.5;
target_neg_samples = round(total_samples * target_neg_proportion / (1 - target_neg_proportion));
synth_samples_needed = target_neg_samples - neg_samples;

if synth_samples_needed <= 0
    error('No additional synthetic samples needed.');
end

% Define key features for synthetic data generation
numerical_features = {'age'};
categorical_features = {'sex', 'nation', 'marriage', 'edu_level', 'politic', ...
    'profession', 'religion', 'c_aac009', 'c_ajc093', 'c_aac183'}; % Include c_aac183 (profession)
date_features = {'c_ajc090'}; % Unemployment time

% Initialize synthetic data table with same structure as negative_class
synth_data = table('Size', [0, width(negative_class)], ...
    'VariableTypes', varfun(@class, negative_class, 'OutputFormat', 'cell'), ...
    'VariableNames', negative_class.Properties.VariableNames);

% Generate synthetic samples
for i = 1:synth_samples_needed
    % Randomly select a negative class sample as a base
    base_idx = randi(neg_samples);
    base_sample = negative_class(base_idx, :);
    
    % Numerical features: Perturb using Gaussian noise
    synth_sample = base_sample;
    for j = 1:length(numerical_features)
        feat = numerical_features{j};
        val = base_sample.(feat);
        if ~ismissing(val)
            % Estimate mean and std from negative class
            feat_vals = negative_class.(feat);
            feat_vals = feat_vals(~ismissing(feat_vals));
            mu = mean(feat_vals);
            sigma = std(feat_vals);
            % Perturb with noise, ensure non-negative and reasonable range
            new_val = round(val + sigma * randn());
            new_val = max(18, min(65, new_val)); % Age range: 18-65
            synth_sample.(feat) = new_val;
        end
    end
    
    % Categorical features: Sample from empirical distribution
    for j = 1:length(categorical_features)
        feat = categorical_features{j};
        val = base_sample.(feat);
        if ~ismissing(val)
            % Get unique values and their frequencies
            feat_vals = negative_class.(feat);
            feat_vals = feat_vals(~ismissing(feat_vals));
            [uniq_vals, ~, idx] = unique(feat_vals);
            probs = accumarray(idx, 1) / length(feat_vals);
            % Sample randomly
            new_val = uniq_vals(randsample(length(uniq_vals), 1, true, probs));
            synth_sample.(feat) = new_val;
        end
    end
    
    % Date features: Perturb unemployment time
    for j = 1:length(date_features)
        feat = date_features{j};
        val = base_sample.(feat);
        if ~ismissing(val)
            % Convert to datetime with specified format
            try
                dt = datetime(val, 'InputFormat', 'yyyy-MM-dd HH:mm:ss', 'Locale', 'zh_CN');
                % Perturb by -30 to +30 days
                new_dt = dt + days(randi([-30, 30]));
                % Ensure within reasonable range (e.g., 2019-2025)
                new_dt = max(datetime('2019-01-01'), min(datetime('2025-04-18'), new_dt));
                synth_sample.(feat) = new_dt;
            catch
                % If conversion fails, use base value
                synth_sample.(feat) = val;
            end
        end
    end
    
    % Generate unique ID and other string fields as cell arrays
    synth_sample.id = {sprintf('SYNTH_%d', i)}; % Cell array to match table format
    synth_sample.people_id = {sprintf('SYNTH_PEOPLE_%d', i)};
    synth_sample.name = {'SYNTH_NAME'};
    
    % Clear employment-related fields to ensure unemployed status
    synth_sample.b_acc030 = {''}; % Employment registration number
    synth_sample.b_acc031 = {''}; % Employment time
    synth_sample.b_aab001 = {''}; % Employer ID
    synth_sample.b_aab004 = {''}; % Employer name
    
    % Sample industry code from original negative class (since b_aab022 is cleared)
    valid_industries = negative_class.b_aab022(~ismissing(negative_class.b_aab022));
    if ~isempty(valid_industries)
        synth_sample.b_aab022 = valid_industries(randi(length(valid_industries)));
    else
        synth_sample.b_aab022 = {''};
    end
    
    % Append to synthetic data
    try
        synth_data = [synth_data; synth_sample];
    catch e
        fprintf('Error concatenating sample %d: %s\n', i, e.message);
        continue; % Skip problematic sample
    end
end

% Save synthetic data to Excel
writetable(synth_data, '负类训练数据.xlsx');
disp(['Generated ', num2str(height(synth_data)), ' synthetic unemployed samples.']);
disp('Saved to 负类训练数据.xlsx');

% --- Feature Extraction ---
% Read the generated synthetic data
synth_data = readtable('负类训练数据.xlsx', 'TreatAsEmpty', {'\N', ''});

% Data Cleaning
% 1. Ensure unemployment time is non-missing
synth_data = synth_data(~ismissing(synth_data.c_ajc090), :);

% 2. Remove records with missing key fields
key_fields = {'age', 'sex', 'edu_level', 'c_aac183', 'b_aab022'};
synth_data = synth_data(~any(ismissing(synth_data(:, key_fields)), 2), :);

% 3. Validate age, gender, education
synth_data = synth_data(synth_data.age >= 0 & synth_data.age <= 100, :);
synth_data = synth_data(ismember(synth_data.sex, [1, 2]), :);
valid_edu = [10,11,12,13,14,15,16,17,18,19,20,21,22,23,28,30,31,32,33,...
             40,41,42,43,44,45,46,47,48,49,50,60,61,62,63,70,71,73,...
             80,81,83,90,91,92,93,99];
synth_data = synth_data(ismember(synth_data.edu_level, valid_edu), :);

% 4. Convert and validate dates
try
    synth_data.c_ajc090 = datetime(synth_data.c_ajc090, 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
catch
    synth_data.c_ajc090 = datetime(synth_data.c_ajc090, 'InputFormat', 'yyyy-MM-dd');
end
synth_data = synth_data(~isnat(synth_data.c_ajc090), :);

% Determine employment status
% All synthetic data should be unemployed (no c_acc028 or b_acc031)
synth_data.employment_status = categorical(false(height(synth_data), 1), [true, false], {'就业', '失业'});

% Check if data remains
if isempty(synth_data)
    error('清洗后没有有效数据！');
end

% Process age: Group into age bins
age_bins = [0, 25, 35, 45, Inf];
age_labels = {'<25', '25-35', '35-45', '>45'};
synth_data.age_group = discretize(synth_data.age, age_bins, 'categorical', age_labels);

% Process gender: Convert to descriptive labels
sex_labels = {'男', '女'};
synth_data.sex = categorical(synth_data.sex, [1, 2], sex_labels);

% Process education: Map to descriptive categories
edu_map = containers.Map(...
    {'10','11','12','13','14','15','16','17','18','19', ... % 研究生及以上
     '20','21','22','23','28', ... % 本科
     '30','31','32','33', ... % 专科
     '40','41','42','43','44','45','46','47','48','49','91','92','93', ... % 中职
     '50','60','61','62','63','70','71','73','80','81','83','90', ... % 高中及以下
     '99'}, ... % 其他
    {'研究生及以上','研究生及以上','研究生及以上','研究生及以上','研究生及以上',...
     '研究生及以上','研究生及以上','研究生及以上','研究生及以上','研究生及以上',...
     '本科','本科','本科','本科','本科',...
     '专科','专科','专科','专科',...
     '中职','中职','中职','中职','中职','中职','中职','中职','中职','中职','中职','中职','中职',...
     '高中及以下','高中及以下','高中及以下','高中及以下','高中及以下',...
     '高中及以下','高中及以下','高中及以下','高中及以下','高中及以下',...
     '高中及以下','高中及以下',...
     '其他'});
synth_data.edu = cell(height(synth_data), 1);
for i = 1:height(synth_data)
    edu_key = num2str(synth_data.edu_level(i));
    if isKey(edu_map, edu_key)
        synth_data.edu{i} = edu_map(edu_key);
    else
        synth_data.edu{i} = '其他';
    end
end
synth_data.edu = categorical(synth_data.edu);

% Process profession: Simplify to top 4 categories + '其他'
synth_data.prof = synth_data.c_aac183;
synth_data.prof = strrep(synth_data.prof, '其他学科', '其他');
prof_categories = unique(synth_data.prof);
if length(prof_categories) > 5
    prof_counts = groupcounts(synth_data, 'prof');
    [~, idx] = sort([prof_counts.GroupCount], 'descend');
    top_profs = prof_categories(idx(1:min(4, length(idx))));
    synth_data.prof(~ismember(synth_data.prof, top_profs)) = {'其他'};
end
synth_data.prof = categorical(synth_data.prof);

% Process industry: Map to five categories
agri_codes = {'A'};
manu_codes = {'C'};
const_mining_codes = {'B', 'E'};
trade_trans_codes = {'F', 'G'};
service_codes = {'D', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'Z'};
synth_data.industry = cell(height(synth_data), 1);
for i = 1:height(synth_data)
    code = string(synth_data.b_aab022{i});
    code_prefix = extractBefore(code, 2);
    if ismember(code_prefix, agri_codes)
        synth_data.industry{i} = '农业及相关产业';
    elseif ismember(code_prefix, manu_codes)
        synth_data.industry{i} = '制造业';
    elseif ismember(code_prefix, const_mining_codes)
        synth_data.industry{i} = '建筑及采矿业';
    elseif ismember(code_prefix, trade_trans_codes)
        synth_data.industry{i} = '批发零售及运输业';
    elseif ismember(code_prefix, service_codes)
        synth_data.industry{i} = '服务业及其他';
    else
        synth_data.industry{i} = '服务业及其他';
    end
end
synth_data.industry = categorical(synth_data.industry);

% Save processed data with extracted features
output_features = synth_data(:, {'id', 'people_id', 'employment_status', 'age_group', 'sex', 'industry', 'prof', 'edu'});
writetable(output_features, '负类训练数据_特征提取.xlsx');
disp('Extracted features saved to 负类训练数据_特征提取.xlsx');

% Summarize employment status
counts = crosstab(synth_data.employment_status);
if length(counts) < 2
    counts(2) = 0;
end

% Display employment status table
col_names = {'就业', '失业'};
row_name = '数量 (人)';
main_title = '表 1 当前就业状态';
top_left_text = '当前就业状态';
data_vals = counts;
disp(main_title);
disp([top_left_text, col_names]);
disp([row_name; num2cell(data_vals)]);